{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:25.125205Z",
     "start_time": "2020-03-07T09:59:25.115829Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# 交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "# one-hot 编码\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T12:05:48.660021Z",
     "start_time": "2020-03-07T12:05:47.677395Z"
    }
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "import re\n",
    "import pandas as pd\n",
    "from pandas import DataFrame as DF\n",
    "import lightgbm as lgb\n",
    "from nltk.corpus import stopwords\n",
    "from scipy.sparse import hstack\n",
    "from hyperopt import STATUS_OK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T12:05:49.640976Z",
     "start_time": "2020-03-07T12:05:49.638655Z"
    }
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data/\"\n",
    "TRAIN_DIR = DATA_DIR + \"train.csv\"\n",
    "TEST_DIR = DATA_DIR + \"test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T12:05:50.257358Z",
     "start_time": "2020-03-07T12:05:50.222902Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size: 7613\n",
      " test size: 3263\n"
     ]
    }
   ],
   "source": [
    "# 训练集, 空格填充空值\n",
    "train = pd.read_csv(TRAIN_DIR, encoding=\"utf-8\").fillna(\" \")\n",
    "# 测试集, 空格填充空值\n",
    "test = pd.read_csv(TEST_DIR, encoding=\"utf-8\").fillna(\" \")\n",
    "\n",
    "print(f\"train size: {len(train)}\\n test size: {len(test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练集去重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:32.998240Z",
     "start_time": "2020-03-07T09:59:32.971938Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7503"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 去重后的索引\n",
    "reserved_train_index = train.text.drop_duplicates().index\n",
    "# 训练集去重\n",
    "train = train.iloc[reserved_train_index, :]\n",
    "\n",
    "len(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 合并训练集和测试集的语料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:34.470994Z",
     "start_time": "2020-03-07T09:59:34.463149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10766"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text = train.text\n",
    "test_text = test.text\n",
    "# 完整语料\n",
    "all_text = pd.concat([train_text, test_text])\n",
    "\n",
    "len(all_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文本预处理\n",
    "1. 去除 URL\n",
    "2. 去除 推特话题\n",
    "3. 在标点符号处分割\n",
    "4. 去掉停用词\n",
    "5. 去掉数字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 去掉缩写"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:36.258398Z",
     "start_time": "2020-03-07T09:59:36.217719Z"
    }
   },
   "outputs": [],
   "source": [
    "def replace_contractions(tweet):\n",
    "    tweet = re.sub(r\"he's\", \"he is\", tweet)\n",
    "    tweet = re.sub(r\"there's\", \"there is\", tweet)\n",
    "    tweet = re.sub(r\"We're\", \"We are\", tweet)\n",
    "    tweet = re.sub(r\"That's\", \"That is\", tweet)\n",
    "    tweet = re.sub(r\"won't\", \"will not\", tweet)\n",
    "    tweet = re.sub(r\"they're\", \"they are\", tweet)\n",
    "    tweet = re.sub(r\"Can't\", \"Cannot\", tweet)\n",
    "    tweet = re.sub(r\"wasn't\", \"was not\", tweet)\n",
    "    tweet = re.sub(r\"don\\x89Ûªt\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"aren't\", \"are not\", tweet)\n",
    "    tweet = re.sub(r\"isn't\", \"is not\", tweet)\n",
    "    tweet = re.sub(r\"What's\", \"What is\", tweet)\n",
    "    tweet = re.sub(r\"haven't\", \"have not\", tweet)\n",
    "    tweet = re.sub(r\"hasn't\", \"has not\", tweet)\n",
    "    tweet = re.sub(r\"There's\", \"There is\", tweet)\n",
    "    tweet = re.sub(r\"He's\", \"He is\", tweet)\n",
    "    tweet = re.sub(r\"It's\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"You're\", \"You are\", tweet)\n",
    "    tweet = re.sub(r\"I'M\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"shouldn't\", \"should not\", tweet)\n",
    "    tweet = re.sub(r\"wouldn't\", \"would not\", tweet)\n",
    "    tweet = re.sub(r\"i'm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"I'm\", \"I am\", tweet)\n",
    "    tweet = re.sub(r\"Isn't\", \"is not\", tweet)\n",
    "    tweet = re.sub(r\"Here's\", \"Here is\", tweet)\n",
    "    tweet = re.sub(r\"you've\", \"you have\", tweet)\n",
    "    tweet = re.sub(r\"you\\x89Ûªve\", \"you have\", tweet)\n",
    "    tweet = re.sub(r\"we're\", \"we are\", tweet)\n",
    "    tweet = re.sub(r\"what's\", \"what is\", tweet)\n",
    "    tweet = re.sub(r\"couldn't\", \"could not\", tweet)\n",
    "    tweet = re.sub(r\"we've\", \"we have\", tweet)\n",
    "    tweet = re.sub(r\"it\\x89Ûªs\", \"it is\", tweet)\n",
    "    tweet = re.sub(r\"doesn\\x89Ûªt\", \"does not\", tweet)\n",
    "    tweet = re.sub(r\"It\\x89Ûªs\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"Here\\x89Ûªs\", \"Here is\", tweet)\n",
    "    tweet = re.sub(r\"who's\", \"who is\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªve\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"y'all\", \"you all\", tweet)\n",
    "    tweet = re.sub(r\"can\\x89Ûªt\", \"cannot\", tweet)\n",
    "    tweet = re.sub(r\"would've\", \"would have\", tweet)\n",
    "    tweet = re.sub(r\"it'll\", \"it will\", tweet)\n",
    "    tweet = re.sub(r\"we'll\", \"we will\", tweet)\n",
    "    tweet = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", tweet)\n",
    "    tweet = re.sub(r\"We've\", \"We have\", tweet)\n",
    "    tweet = re.sub(r\"he'll\", \"he will\", tweet)\n",
    "    tweet = re.sub(r\"Y'all\", \"You all\", tweet)\n",
    "    tweet = re.sub(r\"Weren't\", \"Were not\", tweet)\n",
    "    tweet = re.sub(r\"Didn't\", \"Did not\", tweet)\n",
    "    tweet = re.sub(r\"they'll\", \"they will\", tweet)\n",
    "    tweet = re.sub(r\"they'd\", \"they would\", tweet)\n",
    "    tweet = re.sub(r\"DON'T\", \"DO NOT\", tweet)\n",
    "    tweet = re.sub(r\"That\\x89Ûªs\", \"That is\", tweet)\n",
    "    tweet = re.sub(r\"they've\", \"they have\", tweet)\n",
    "    tweet = re.sub(r\"i'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"should've\", \"should have\", tweet)\n",
    "    tweet = re.sub(r\"You\\x89Ûªre\", \"You are\", tweet)\n",
    "    tweet = re.sub(r\"where's\", \"where is\", tweet)\n",
    "    tweet = re.sub(r\"Don\\x89Ûªt\", \"Do not\", tweet)\n",
    "    tweet = re.sub(r\"we'd\", \"we would\", tweet)\n",
    "    tweet = re.sub(r\"i'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"weren't\", \"were not\", tweet)\n",
    "    tweet = re.sub(r\"They're\", \"They are\", tweet)\n",
    "    tweet = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", tweet)\n",
    "    tweet = re.sub(r\"you\\x89Ûªll\", \"you will\", tweet)\n",
    "    tweet = re.sub(r\"I\\x89Ûªd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"let's\", \"let us\", tweet)\n",
    "    tweet = re.sub(r\"it's\", \"it is\", tweet)\n",
    "    tweet = re.sub(r\"can't\", \"cannot\", tweet)\n",
    "    tweet = re.sub(r\"don't\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"you're\", \"you are\", tweet)\n",
    "    tweet = re.sub(r\"i've\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"that's\", \"that is\", tweet)\n",
    "    tweet = re.sub(r\"i'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"doesn't\", \"does not\", tweet)\n",
    "    tweet = re.sub(r\"i'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"didn't\", \"did not\", tweet)\n",
    "    tweet = re.sub(r\"ain't\", \"am not\", tweet)\n",
    "    tweet = re.sub(r\"you'll\", \"you will\", tweet)\n",
    "    tweet = re.sub(r\"I've\", \"I have\", tweet)\n",
    "    tweet = re.sub(r\"Don't\", \"do not\", tweet)\n",
    "    tweet = re.sub(r\"I'll\", \"I will\", tweet)\n",
    "    tweet = re.sub(r\"I'd\", \"I would\", tweet)\n",
    "    tweet = re.sub(r\"Let's\", \"Let us\", tweet)\n",
    "    tweet = re.sub(r\"you'd\", \"You would\", tweet)\n",
    "    tweet = re.sub(r\"It's\", \"It is\", tweet)\n",
    "    tweet = re.sub(r\"Ain't\", \"am not\", tweet)\n",
    "    tweet = re.sub(r\"Haven't\", \"Have not\", tweet)\n",
    "    tweet = re.sub(r\"Could've\", \"Could have\", tweet)\n",
    "    tweet = re.sub(r\"youve\", \"you have\", tweet)  \n",
    "    tweet = re.sub(r\"donå«t\", \"do not\", tweet) \n",
    "    return tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:02:25.255754Z",
     "start_time": "2020-03-07T04:02:25.253648Z"
    }
   },
   "outputs": [],
   "source": [
    "# 没有装nltk 的 stopwords 的话\n",
    "# import nltk\n",
    "# nltk.download(\"stopwords\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 停用词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:26.051610Z",
     "start_time": "2020-03-07T04:49:26.048948Z"
    }
   },
   "outputs": [],
   "source": [
    "custom_stopwords = []\n",
    "\n",
    "stopword_set = stopwords.words(\"english\") + custom_stopwords + [\"url\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 去掉 emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:40.278745Z",
     "start_time": "2020-03-07T09:59:40.276128Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 完整的预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:41.454544Z",
     "start_time": "2020-03-07T09:59:41.449744Z"
    }
   },
   "outputs": [],
   "source": [
    "def preProcess(iter):\n",
    "    #https://www.kaggle.com/shahules/basic-eda-cleaning-and-glove\n",
    "        # remove extra space\n",
    "        regex_ws=re.compile(\"\\s+\")\n",
    "        ret=regex_ws.sub(\" \",iter)\n",
    "        \n",
    "        ret=ret.replace(\"&amp;\",\"&\").replace(\"&lt;\",\"<\").replace(\"&gt;\",\">\")\n",
    "        \n",
    "        \n",
    "        #Replace slang words\n",
    "        #for key in abbreviations.keys():\n",
    "        #    ret=ret.replace(key,abbreviations[key])\n",
    "        \n",
    "        #Replace URL\n",
    "        regexp=\"(https?:\\/\\/(?:www\\.|(?!www)|(?:xmlns\\.))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|www\\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9]+\\.[^\\s]{2,}|www\\.[a-zA-Z0-9]+\\.[^\\s]{2,})\"\n",
    "        ret=re.sub(regexp,\"url\",ret)\n",
    "        \n",
    "        #replace @addresses\n",
    "        regexp='@[A-z0-9_]+'\n",
    "        ret=re.sub(regexp,\"@twitterhandle\",ret)\n",
    "        \n",
    "        ret=remove_emoji(ret)\n",
    "        #ret=replace_contractions(ret)\n",
    "        #Split on punctuations\n",
    "        ret1=re.split(\"[,_, \\<>!\\?\\.:\\n\\\"=*/]+\",ret)\n",
    "        \n",
    "        #Remove Stopwords\n",
    "        ret2=[word for word in ret1 if word not in stopword_set]\n",
    "        ret2=\" \".join(ret2)\n",
    "        \n",
    "        #Remove  numbers\n",
    "        ret2=re.sub(r\"(\\s\\d+)\",\" \",ret2)\n",
    "                \n",
    "        #STEM TEXT\n",
    "        #ret3=stem_text(strip_punctuation(ret2))\n",
    "    \n",
    "        return ret2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tf-idf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以单词为单位向量化 ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:42.943245Z",
     "start_time": "2020-03-07T09:59:42.938188Z"
    }
   },
   "outputs": [],
   "source": [
    "word_vectorizer = TfidfVectorizer(\n",
    "    sublinear_tf=True,\n",
    "    strip_accents='unicode',\n",
    "    analyzer='word',\n",
    "    token_pattern=r'\\w{1,}',\n",
    "    stop_words='english',\n",
    "    ngram_range=(1, 1),\n",
    "    norm='l2',\n",
    "    min_df=0,\n",
    "    smooth_idf=False,\n",
    "    preprocessor=preProcess,\n",
    "    max_features=15000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T09:59:48.059381Z",
     "start_time": "2020-03-07T09:59:47.928602Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'stopword_set' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-4cce77954bbe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mword_vectorizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtrain_word_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword_vectorizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtest_word_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword_vectorizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m   1834\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1835\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_warn_for_unused_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1836\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1837\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1838\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m   1218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1219\u001b[0m         vocabulary, X = self._count_vocab(raw_documents,\n\u001b[0;32m-> 1220\u001b[0;31m                                           self.fixed_vocabulary_)\n\u001b[0m\u001b[1;32m   1221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1222\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36m_count_vocab\u001b[0;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[1;32m   1129\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mraw_documents\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1130\u001b[0m             \u001b[0mfeature_counter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1131\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mfeature\u001b[0m \u001b[0;32min\u001b[0m \u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1132\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1133\u001b[0m                     \u001b[0mfeature_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvocabulary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36m_analyze\u001b[0;34m(doc, analyzer, tokenizer, ngrams, preprocessor, decoder, stop_words)\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpreprocessor\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m             \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocessor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-0201d67eaf6a>\u001b[0m in \u001b[0;36mpreProcess\u001b[0;34m(iter)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0;31m#Remove Stopwords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mret2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mret1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstopword_set\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0mret2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-0201d67eaf6a>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0;31m#Remove Stopwords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mret2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mret1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstopword_set\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0mret2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'stopword_set' is not defined"
     ]
    }
   ],
   "source": [
    "word_vectorizer.fit(all_text)\n",
    "\n",
    "train_word_features = word_vectorizer.transform(train_text)\n",
    "\n",
    "test_word_features = word_vectorizer.transform(test_text)\n",
    "\n",
    "train_word_features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以字符为单位向量化 ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:34.044906Z",
     "start_time": "2020-03-07T04:49:34.042190Z"
    }
   },
   "outputs": [],
   "source": [
    "char_vectorizer = TfidfVectorizer(\n",
    "    sublinear_tf=True,\n",
    "    strip_accents='unicode',\n",
    "    analyzer='char',\n",
    "    ngram_range=(2, 6),    # 词袋\n",
    "    norm='l2',\n",
    "    min_df=0,\n",
    "    smooth_idf=False,preprocessor=preProcess,\n",
    "    max_features=30000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:42.868505Z",
     "start_time": "2020-03-07T04:49:36.036440Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7503, 30000)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_vectorizer.fit(all_text)\n",
    "\n",
    "train_char_features = char_vectorizer.transform(train_text)\n",
    "\n",
    "test_char_features = char_vectorizer.transform(test_text)\n",
    "\n",
    "train_char_features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## one-hot 编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:44.151823Z",
     "start_time": "2020-03-07T04:49:44.144157Z"
    }
   },
   "outputs": [],
   "source": [
    "# one-hot 编码\n",
    "enc = OneHotEncoder(handle_unknown='ignore')\n",
    "\n",
    "train_keyword_features = enc.fit_transform(train['keyword'].to_numpy().reshape(-1,1))\n",
    "test_keyword_features = enc.transform(test['keyword'].to_numpy().reshape(-1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 组合特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:45.736477Z",
     "start_time": "2020-03-07T04:49:45.571308Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 转为稀疏矩阵\n",
    "train_features = hstack([train_char_features, train_word_features]).tocsr()\n",
    "\n",
    "del train_word_features, train_char_features\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:47.640757Z",
     "start_time": "2020-03-07T04:49:47.524640Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_features = hstack([test_char_features, test_word_features]).tocsr()\n",
    "\n",
    "del test_char_features, test_word_features\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:48.479524Z",
     "start_time": "2020-03-07T04:49:48.476959Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train shape: (7503, 45000)\n",
      " test shape: (3263, 45000)\n"
     ]
    }
   ],
   "source": [
    "print(f\"train shape: {train_features.shape}\\n test shape: {test_features.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lightGBM 分类器 + hyperopt 贝叶斯优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:49:50.196028Z",
     "start_time": "2020-03-07T04:49:50.192765Z"
    }
   },
   "outputs": [],
   "source": [
    "# 5 折交叉验证\n",
    "N_FOLDS = 5\n",
    "# 训练集\n",
    "train_set = lgb.Dataset(train_features, train.target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自定义目标函数\n",
    "目标函数应在优化中 **最小化**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:16:14.358892Z",
     "start_time": "2020-03-07T05:16:14.351788Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def f1_metric(ytrue, preds):\n",
    "    \"\"\" 自定义的 F1 score\n",
    "    \"\"\"\n",
    "    return 'f1', f1_score((preds>=0.5).astype('int'), ytrue, average='macro'), True\n",
    "\n",
    "# 最小化的目标函数\n",
    "def objective(params, n_folds=N_FOLDS):\n",
    "    \"\"\" 自动调参中的目标函数\n",
    "    :param params: 超参数\n",
    "    :n_folds: int, 折数\n",
    "    \"\"\"\n",
    "    params[\"num_leaves\"] = int(params[\"num_leaves\"])\n",
    "    params[\"min_data_in_leaf\"] = int(params[\"min_child_samples\"])\n",
    "    params[\"max_depth\"] = int(params[\"max_depth\"])\n",
    "    cv_results = lgb.cv(\n",
    "        params=params, train_set=train_set,\n",
    "        nfold=n_folds, num_boost_round=100,\n",
    "        early_stopping_rounds=10, feval=lgb_f1_score,    # lgb 不提供 F1, 此处以 auc 为评价标准\n",
    "        seed=5000\n",
    "    )\n",
    "    \n",
    "#     best_score = max(cv_results[\"auc-mean\"])\n",
    "    best_score = max(cv_results[\"f1-mean\"])\n",
    "    # 损失应最小化\n",
    "    loss = 1 - best_score\n",
    "    return {\"loss\": loss, \"params\": params, \"status\": STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:51:41.217438Z",
     "start_time": "2020-03-07T04:51:41.213852Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
       "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
       "               random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看分类器的所有超参数\n",
    "lgb.LGBMClassifier()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义搜索空间\n",
    "- choice: 类别变量\n",
    "- quniform: 离散均匀分布（在整数空间上均匀分布）\n",
    "- uniform: 连续均匀分布（在浮点数空间上均匀分布）\n",
    "- loguniform: 连续对数均匀分布（在浮点数空间的对数尺度上均匀分布）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:16:18.559762Z",
     "start_time": "2020-03-07T05:16:18.553300Z"
    }
   },
   "outputs": [],
   "source": [
    "from hyperopt import hp\n",
    "\n",
    "# 搜索空间\n",
    "space = {\n",
    "    'class_weight': hp.choice('class_weight', [None, 'balanced']),\n",
    "    # 梯度提升树类型, 使用条件域, 即 goss 类型的梯度提升树不使用降采样\n",
    "#     'boosting_type': hp.choice('boosting_type', \n",
    "#                                [{'boosting_type': 'gbdt', \n",
    "#                                     'subsample': hp.uniform('gdbt_subsample', 0.5, 1)}, \n",
    "#                                  {'boosting_type': 'dart', \n",
    "#                                      'subsample': hp.uniform('dart_subsample', 0.5, 1)},\n",
    "#                                  {'boosting_type': 'goss'}]),\n",
    "    # lgb 的关键, 叶子节点数\n",
    "    'num_leaves': hp.quniform('num_leaves', 25, 80, 1),\n",
    "    'max_depth': hp.quniform('max_depth', 5, 8, 1),\n",
    "    # lgb 关键, 学习率\n",
    "    'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.15)),\n",
    "#     'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),\n",
    "    # 以下三个都是放止过拟合\n",
    "    'min_child_samples': hp.quniform('min_child_samples', 20, 100, 5),\n",
    "    'reg_alpha': hp.uniform('reg_alpha', 0.0, 0.1),  # L1 正则化\n",
    "    'reg_lambda': hp.uniform('reg_lambda', 0.0, 0.1),  # L2 正则化\n",
    "#     'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选择优化算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:30:11.905136Z",
     "start_time": "2020-03-07T05:30:11.901605Z"
    }
   },
   "outputs": [],
   "source": [
    "from hyperopt import tpe\n",
    "from hyperopt import atpe\n",
    "# 使用树形 Parzen 评估器 (TPE)\n",
    "tpe_algorithm = atpe.suggest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 存储结果的历史信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:30:14.783266Z",
     "start_time": "2020-03-07T05:30:14.780885Z"
    }
   },
   "outputs": [],
   "source": [
    "from hyperopt import Trials\n",
    "\n",
    "bayes_trials = Trials()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化\n",
    " 0.29209872275578785"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:52:16.052484Z",
     "start_time": "2020-03-07T05:30:23.689874Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [21:52<00:00, 13.12s/trial, best loss: 0.29004972203458623]\n"
     ]
    }
   ],
   "source": [
    "from hyperopt import fmin\n",
    "\n",
    "MAX_EVALS = 100\n",
    "\n",
    "# 优化\n",
    "best = fmin(\n",
    "    fn=objective, \n",
    "    space=space,\n",
    "    algo=tpe_algorithm,\n",
    "    max_evals=MAX_EVALS,\n",
    "    trials=bayes_trials\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 最优参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:52:42.282657Z",
     "start_time": "2020-03-07T05:52:42.279284Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_weight': 1,\n",
       " 'learning_rate': 0.1494808274171447,\n",
       " 'max_depth': 8.0,\n",
       " 'min_child_samples': 55.0,\n",
       " 'num_leaves': 31.0,\n",
       " 'reg_alpha': 0.09228096612486511,\n",
       " 'reg_lambda': 0.02623513537101767}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:05.224647Z",
     "start_time": "2020-03-07T05:53:05.222052Z"
    }
   },
   "outputs": [],
   "source": [
    "param_names = [\"max_depth\", \"min_child_samples\", \"num_leaves\"]\n",
    "\n",
    "for name in param_names:\n",
    "    best[name] = int(best[name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:06.401080Z",
     "start_time": "2020-03-07T05:53:06.397545Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_weight': 1,\n",
       " 'learning_rate': 0.1494808274171447,\n",
       " 'max_depth': 8,\n",
       " 'min_child_samples': 55,\n",
       " 'num_leaves': 31,\n",
       " 'reg_alpha': 0.09228096612486511,\n",
       " 'reg_lambda': 0.02623513537101767}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用最佳参数在完整训练集上训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:13.855459Z",
     "start_time": "2020-03-07T05:53:10.961302Z"
    }
   },
   "outputs": [],
   "source": [
    "clf = lgb.train(\n",
    "    params=best,\n",
    "    train_set=train_set,\n",
    "    num_boost_round=90,\n",
    "    feval=lgb_f1_score    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb.cv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T04:23:42.918290Z",
     "start_time": "2020-03-07T04:23:32.034567Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               importance_type='split', learning_rate=0.07271150650964049,\n",
       "               max_depth=-1, metric='auc', min_child_samples=35,\n",
       "               min_child_weight=0.001, min_split_gain=0.0, n_estimators=100,\n",
       "               n_jobs=4, num_leaves=35, objective='binary', random_state=None,\n",
       "               reg_alpha=0.06788694904077003, reg_lambda=0.3843472676294802,\n",
       "               silent=True, subsample=1.0, subsample_for_bin=200000,\n",
       "               subsample_freq=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(train_features, train.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:00:24.726482Z",
     "start_time": "2020-03-07T05:00:24.707787Z"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "estimator should be an estimator implementing 'fit' method, <lightgbm.basic.Booster object at 0x7f958220b518> was passed",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-a4fc5cf5b058>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcross_val_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"f1\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_val_score\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, error_score)\u001b[0m\n\u001b[1;32m    381\u001b[0m     \"\"\"\n\u001b[1;32m    382\u001b[0m     \u001b[0;31m# To ensure multimetric format is not supported\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 383\u001b[0;31m     \u001b[0mscorer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_scoring\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mscoring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    384\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    385\u001b[0m     cv_results = cross_validate(estimator=estimator, X=X, y=y, groups=groups,\n",
      "\u001b[0;32m~/miniconda3/envs/torch/lib/python3.6/site-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36mcheck_scoring\u001b[0;34m(estimator, scoring, allow_none)\u001b[0m\n\u001b[1;32m    399\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m         raise TypeError(\"estimator should be an estimator implementing \"\n\u001b[0;32m--> 401\u001b[0;31m                         \"'fit' method, %r was passed\" % estimator)\n\u001b[0m\u001b[1;32m    402\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscoring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mget_scorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscoring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: estimator should be an estimator implementing 'fit' method, <lightgbm.basic.Booster object at 0x7f958220b518> was passed"
     ]
    }
   ],
   "source": [
    "cross_val_score(clf, train_features, train.target, scoring=\"f1\", cv=5).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-06T13:24:49.250502Z",
     "start_time": "2020-03-06T13:24:49.247432Z"
    }
   },
   "outputs": [],
   "source": [
    "clf = lgb.sklearn.LGBMClassifier(\n",
    "    objective=\"binary\",  # 二分类\n",
    "    metric=\"f1\",        # 评价标准 f1\n",
    "    n_estimators=95,  # 迭代次数\n",
    "    silent=True,      # 不输出\n",
    "    n_jobs=4,        # 进程数\n",
    "    reg_lambda=0.05 # L2 正则化\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-06T13:33:55.010230Z",
     "start_time": "2020-03-06T13:31:24.718704Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5976658826380282"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 交叉验证查看这组参数的 f1\n",
    "cross_val_score(clf, train_features, train.target, scoring=\"f1\", cv=5, n_jobs=4).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-06T13:14:40.702806Z",
     "start_time": "2020-03-06T13:14:28.658168Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
       "               metric='f1', min_child_samples=20, min_child_weight=0.001,\n",
       "               min_split_gain=0.0, n_estimators=95, n_jobs=4, num_leaves=31,\n",
       "               objective='binary', random_state=None, reg_alpha=0.0,\n",
       "               reg_lambda=0.05, silent=True, subsample=1.0,\n",
       "               subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 在训练集上训练\n",
    "# 注意 : 交叉验证并不算模型训练\n",
    "clf.fit(train_features, train.target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-06T12:57:40.491347Z",
     "start_time": "2020-03-06T12:57:40.487601Z"
    }
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"num_leaves\": [26, 28, 30, 32, 34],\n",
    "    \"max_depth\": [6, 7, 8],\n",
    "    \"learning_rate\": [0.08, 0.1, 0.12]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = GridSearchCV(clf, param_grid=params, scoring=\"f1\", cv=3)\n",
    "\n",
    "grid.fit(train_features, train.target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在测试集上预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:25.351615Z",
     "start_time": "2020-03-07T05:53:25.303121Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3263,)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(test_features)\n",
    "\n",
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:26.488530Z",
     "start_time": "2020-03-07T05:53:26.485098Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.68807538, 0.46710538, 0.40464456, 0.7604146 , 0.84055125])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:45.777201Z",
     "start_time": "2020-03-07T05:53:45.771904Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = list(map(lambda x: 1 if x > 0.5 else 0, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:46.342842Z",
     "start_time": "2020-03-07T05:53:46.339272Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 0, 0, 1, 1]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:08:01.660750Z",
     "start_time": "2020-03-07T05:08:01.657180Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0], dtype=int32)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "y_pred.astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-07T05:53:52.928948Z",
     "start_time": "2020-03-07T05:53:52.913123Z"
    }
   },
   "outputs": [],
   "source": [
    "df = DF({\"id\": test.id, \"target\": y_pred})\n",
    "\n",
    "df.to_csv(\"./submit3.7.csv\", index=False, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试集 ```F1``` 0.80777"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "349.091px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "503.722px",
    "left": "1383.45px",
    "right": "20px",
    "top": "101.986px",
    "width": "350px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
